package com.kavian.test.search.proxy;

import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.Span;
import org.htmlparser.util.NodeList;

import com.kavian.common.util.UuidUtil;
import com.kavian.test.db.JdbcPool;

/**
 * 爬取代理IP
 * 
 * @author kavian
 * 
 */
public class GetProxyIp {

	/**
	 * URL地址
	 */
	// final static String URL = "http://kavian.iteye.com/blog/1554353";
	final static String URL = "http://www.proxy360.cn/default.aspx";

	HttpClient httpclient;
	HttpGet httpGet;

	public GetProxyIp(String url) {
		initialHttpClient(url);
	}

	/**
	 * 初始化HttpClient
	 */
	private void initialHttpClient(String url) {
		try {
			// 生成 HttpClinet 对象
			httpclient = new DefaultHttpClient();
			// 生成 GetMethod 对象
			httpGet = new HttpGet(url);
			// 设置请求头，这里很关键，细节在注意事项中描述
			httpGet.setHeader("user-agent",
					"Mozilla/5.0 (Windows NT 5.1; rv:8.0.1) Gecko/20100101");
			httpGet.setHeader("accept", "*/*");
			httpGet.setHeader("accept-language", "zh-CN");
			httpGet.setHeader("accept-encoding", "utf-8");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	/**
	 * 反回内容集合
	 * 
	 * @return
	 */
	public List<ProxyIp> getContent() {
		List<ProxyIp> list = new ArrayList<ProxyIp>();
		try {
			HttpResponse response = httpclient.execute(httpGet);
			HttpEntity entity = response.getEntity();
			String content = EntityUtils.toString(entity);

			Parser divParser = new Parser(content);
			NodeFilter proxylistitem = new AndFilter(new NodeClassFilter(
					Div.class),
					new HasAttributeFilter("class", "proxylistitem"));
			NodeList divList = divParser
					.extractAllNodesThatMatch(proxylistitem);

			for (int i = 0; i < divList.size(); i++) {
				String divHtml = divList.elementAt(i).toHtml();
				Parser spanParser = new Parser(divHtml);

				NodeFilter tbBottomLine = new AndFilter(new NodeClassFilter(
						Span.class), new HasAttributeFilter("class",
						"tbBottomLine"));
				NodeList spanList = spanParser
						.extractAllNodesThatMatch(tbBottomLine);

				ProxyIp proxyIp = new ProxyIp();
				for (int j = 0; j < spanList.size(); j++) {
					String spanHtml = spanList.elementAt(j).toPlainTextString();
//					System.out.println(spanHtml);
					 if(j == 0){
					 proxyIp.setIp(spanHtml.trim());
					 }else if(j == 1){
					 proxyIp.setProt(spanHtml.trim());
					 }
				}
//				System.out.println("--------");
				list.add(proxyIp);
			}

		} catch (Exception e) {
			e.printStackTrace();
		}

		return list;
	}

	public static void main(String[] args) {
		GetProxyIp proxy = new GetProxyIp(URL);
		List<ProxyIp> list = proxy.getContent();
		JdbcPool jdbc = new JdbcPool();
		String sql = "INSERT INTO proxy_ip(id,ip,port,is_valid) VALUES(?,?,?,?)";
		if (null != list && list.size() != 0) {
			for (ProxyIp proxyIp : list) {
				//System.out.println(proxyIp.getIp() + ":" + proxyIp.getProt());
				Object[] obj = {UuidUtil.getUuid(),proxyIp.getIp(),proxyIp.getProt(),1};
				jdbc.executeSql(sql, obj);
			}
		}
		
		jdbc.close();
	}
}